{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from malaya.train.model.bigbird import modeling, utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "bert_config = {\n",
    "    'attention_probs_dropout_prob': 0.1,\n",
    "    'hidden_act': 'gelu',\n",
    "    'hidden_dropout_prob': 0.1,\n",
    "    'hidden_size': 256,\n",
    "    'initializer_range': 0.02,\n",
    "    'intermediate_size': 1024,\n",
    "    'max_position_embeddings': 2048,\n",
    "    'max_encoder_length': 1024,\n",
    "    'max_decoder_length': 1024,\n",
    "    'num_attention_heads': 4,\n",
    "    'num_hidden_layers': 2,\n",
    "    'type_vocab_size': 2,\n",
    "    'scope': 'bert',\n",
    "    'use_bias': True,\n",
    "    'rescale_embedding': False,\n",
    "    'vocab_model_file': None,\n",
    "    'attention_type': 'block_sparse',\n",
    "    'block_size': 16,\n",
    "    'num_rand_blocks': 3,\n",
    "    'vocab_size': 32000,\n",
    "    'couple_encoder_decoder': False,\n",
    "    'beam_size': 1,\n",
    "    'alpha': 0.0,\n",
    "    'label_smoothing': 0.1,\n",
    "    'norm_type': 'postnorm',\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sentencepiece as spm\n",
    "\n",
    "vocab = 'sp10m.cased.translation.model'\n",
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(vocab)\n",
    "\n",
    "class Encoder:\n",
    "    def __init__(self, sp):\n",
    "        self.sp = sp\n",
    "    \n",
    "    def encode(self, s):\n",
    "        return self.sp.EncodeAsIds(s) + [1]\n",
    "    \n",
    "    def decode(self, ids, strip_extraneous=False):\n",
    "        return self.sp.DecodeIds(list(ids))\n",
    "    \n",
    "encoder = Encoder(sp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = modeling.TransformerModel(bert_config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = tf.placeholder(tf.int32, [None, None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/malaya/Malaya/malaya/train/model/bigbird/modeling.py:226: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "((<tf.Tensor 'bert/log_probs:0' shape=(?, 1024) dtype=float32>,\n",
       "  <tf.Tensor 'bert/logits:0' shape=(?, 1024, 32000) dtype=float32>,\n",
       "  <tf.Tensor 'bert/while/Exit_1:0' shape=(?, 1024) dtype=int32>),\n",
       " <tf.Tensor 'bert/encoder/layer_1/output/LayerNorm/batchnorm/add_1:0' shape=(?, 1024, 256) dtype=float32>)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r = model(X, training = False)\n",
    "r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'logits:0' shape=(?, 1024) dtype=int32>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logits = tf.identity(r[0][2], name = 'logits')\n",
    "logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bigbird-small-ms-en/model.ckpt-470000'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ckpt_path = tf.train.latest_checkpoint('bigbird-small-ms-en')\n",
    "ckpt_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.InteractiveSession()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bigbird-small-ms-en/model.ckpt-470000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from bigbird-small-ms-en/model.ckpt-470000\n"
     ]
    }
   ],
   "source": [
    "saver = tf.train.Saver()\n",
    "saver.restore(sess, ckpt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad_sequences = tf.keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "from unidecode import unidecode\n",
    "\n",
    "def cleaning(string):\n",
    "    return re.sub(r'[ ]+', ' ', unidecode(string.replace('\\n', ' '))).strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'KUALA LUMPUR: Perseteruan antara dua bekas Perdana Menteri, Tun Dr Mahathir Mohamad dan Datuk Seri Najib Tun Razak belum ada penghujungnya dengan masing-masing berbalas kenyataan di media sosial. Selepas Najib menyanggah kenyataan tidak campur tangan dalam badan kehakiman negara dan mentertawakannya, Dr Mahathir membalasnya dengan meminta Ahli Parlimen Pekan itu memberi perhatian kepada kes 1Malaysia Development Berhad (1MDB). Dr Mahathir juga secara sinis berkata, jika Najib boleh mengingati isu yang berlaku pada 1987 dan 1988 - isu pemilihan UMNO dan pengharaman parti itu, Najib juga boleh mengingati peristiwa dia ingin mandikan keris dengan darah. \"Saya rasa Najib tak payah campur tangan dengan tuduhan terhadap saya. Dia harus fokus kes curi duit rakyat berbilion-bilion dalam 1MDB. \"Dia juga perlu bagi perhatian saman Tommy Thomas yang kait dia dengan pembunuhan Altantuya. Lagipun, kalau isu yang berlaku 1987/88, Najib boleh ingat (peristiwa) yang dia \\'hunus\\' keris,\" kata Dr Mahathir. Sebelum ini, Najib menyanggah dakwaan Dr Mahathir yang mendakwa pembatalan pendaftaran UMNO pada 1998 sebagai bukti tidak campur tangan dalam badan kehakiman negara. Menyokong hujahnya, Najib berkongsi apa yang berlaku pada tahun tersebut hingga menyebabkan UMNO diharamkan dan tertubuhnya UMNO baharu.'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string = \"\"\"\n",
    "KUALA LUMPUR: Perseteruan antara dua bekas Perdana Menteri, Tun Dr Mahathir Mohamad dan Datuk Seri Najib Tun Razak belum ada penghujungnya dengan masing-masing berbalas kenyataan di media sosial.\n",
    "Selepas Najib menyanggah kenyataan tidak campur tangan dalam badan kehakiman negara dan mentertawakannya, Dr Mahathir membalasnya dengan meminta Ahli Parlimen Pekan itu memberi perhatian kepada kes 1Malaysia Development Berhad (1MDB).\n",
    "Dr Mahathir juga secara sinis berkata, jika Najib boleh mengingati isu yang berlaku pada 1987 dan 1988 - isu pemilihan UMNO dan pengharaman parti itu, Najib juga boleh mengingati peristiwa dia ingin mandikan keris dengan darah.\n",
    "\"Saya rasa Najib tak payah campur tangan dengan tuduhan terhadap saya. Dia harus fokus kes curi duit rakyat berbilion-bilion dalam 1MDB.\n",
    "\"Dia juga perlu bagi perhatian saman Tommy Thomas yang kait dia dengan pembunuhan Altantuya. Lagipun, kalau isu yang berlaku 1987/88, Najib boleh ingat (peristiwa) yang dia 'hunus' keris,\" kata Dr Mahathir.\n",
    "Sebelum ini, Najib menyanggah dakwaan Dr Mahathir yang mendakwa pembatalan pendaftaran UMNO pada 1998 sebagai bukti tidak campur tangan dalam badan kehakiman negara.\n",
    "Menyokong hujahnya, Najib berkongsi apa yang berlaku pada tahun tersebut hingga menyebabkan UMNO diharamkan dan tertubuhnya UMNO baharu.\n",
    "\"\"\"\n",
    "cleaning(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "string3 = \"\"\"\n",
    "Penubuhan universiti sukan seperti diutarakan Ketua Unit Sukan Kementerian Pengajian Tinggi, Dr Pekan Ramli dan disokong Pakar Pembangunan Sukan dan Reakreasi Luar, Universiti Pendidikan Sultan Idris (UPSI), Prof Dr Md Amin Md Taaf seperti disiarkan akhbar ini, memberikan sinar harapan kepada kewujudan institusi sedemikian.\n",
    "\n",
    "Ia menjadi impian atlet negara untuk mengejar kejayaan dalam bidang sukan dan kecemerlangan dalam akademik untuk menjamin masa depan lebih baik apabila bersara daripada arena sukan kelak.\n",
    "\n",
    "Pelbagai pandangan, idea, kaedah, bukti dan cadangan dilontarkan pakar berikutan pentingnya universiti sukan yang akan memberi impak besar sama ada pada peringkat kebangsaan mahupun antarabangsa.\n",
    "\n",
    "Negara lain sudah lama meraih laba dengan kewujudan universiti sukan seperti China, Korea, Japan, Taiwan, India dan Vietnam. Mereka menghasilkan atlet universiti yang mempamerkan keputusan cemerlang pada peringkat tinggi seperti Sukan Olimpik, Kejohanan Dunia dan Sukan Asia.\n",
    "\n",
    "Justeru, kejayaan mereka perlu dijadikan rujukan demi memajukan sukan tanah air. Jika kita merujuk pendekatan Asia, kewujudan universiti sukan penting dan memberi kesan positif dalam melonjakkan prestasi sukan lebih optimum.\n",
    "\n",
    "Namun, jika kita melihat pendekatan Eropah, universiti sukan bukan antara organisasi atau institusi penting yang diberi perhatian dalam menyumbang kepada pemenang pingat.\n",
    "\n",
    "Antara isu dalam universiti sukan ialah kos tinggi, lokasi, prasarana sukan, pertindihan kursus dengan universiti sedia ada dan impak terhadap dunia sukan negara hingga mengundang persoalan kewajaran dan kerelevanan penubuhannya.\n",
    "\n",
    "Namun sebagai bekas atlet memanah negara dan Olympian (OLY) di Sukan Olimpik 2004 di Athens, Greece serta bekas pelajar Sekolah Sukan Bukit Jalil hingga berjaya dalam dunia akademik, saya mendapati terdapat beberapa faktor sering menjadi halangan dalam rutin harian mereka.\n",
    "\n",
    "Antaranya, faktor masa yang terpaksa bergegas menghadiri kuliah selepas tamat sesi latihan yang mengambil masa 15 hingga 20 minit dengan menunggang motosikal; kereta (20-30 minit) atau pengangkutan disediakan Majlis Sukan Negara (MSN) ke Universiti Putra Malaysia (UPM).\n",
    "\n",
    "Jika mereka menuntut di Universiti Teknologi MARA (UiTM) atau Universiti Malaya (UM), ia mungkin lebih lama.\n",
    "\n",
    "Walaupun di universiti tersedia dengan kemudahan kolej dan kemudahan sukan, mereka memilih pulang ke MSN untuk menjalani latihan bersama pasukan dan jurulatih di padang atau gelanggang latihan rasmi.\n",
    "\n",
    "Ini berlanjutan selagi bergelar atlet negara yang perlu memastikan prestasi sentiasa meningkat dari semasa ke semasa tanpa mengabaikan tugas sebagai pelajar.\n",
    "\n",
    "Alangkah baiknya jika sebahagian Sekolah Sukan Bukit Jalil itu sendiri dijadikan Kolej atau Universiti Sukan Malaysia kerana lengkap dari segi kemudahan prasarana sukannya dan proses pengajaran dan pembelajaran (PdP) dalam bidang Sains Sukan, Kejurulatihan, Pendidikan Jasmani dan setaraf dengannya.\n",
    "\n",
    "Pengambilan setiap semester pula hanya terhad kepada atlet berstatus kebangsaan dan antarabangsa sahaja supaya hasrat melahirkan lebih ramai atlet bertaraf Olimpik mudah direalisasikan.\n",
    "\n",
    "Contohnya, bekas atlet lompat bergalah negara, Roslinda Samsu yang juga pemenang pingat perak Sukan Asia Doha 2006 dan Penerima Anugerah Khas Majlis Anugerah Sukan KPT 2012, terpaksa mengambil masa lebih kurang sembilan tahun untuk menamatkan ijazah Sarjana Muda Pendidikan Jasmani di UPM sepanjang 14 tahun terbabit dalam sukan olahraga.\n",
    "\n",
    "Sepanjang tempoh bergelar atlet kebangsaan dan mahasiswa, beliau juga memenangi pingat Emas Sukan SEA empat siri berturut-turut pada 2005, 2007, 2009 dan 2011.\n",
    "\n",
    "Begitu juga atlet kebangsaan seperti Leong Mun Yee (UPM); Pandalela Renong (UM); Bryan Nickson Lomas (UM); Cheng Chu Sian (UPM); Marbawi Sulaiman (UiTM) dan Norasheela Khalid (UPM).\n",
    "\n",
    "Jika disenaraikan, mungkin lebih ramai lagi. Namun, pernah terlintas di fikiran mengapa hanya atlet dari sukan terjun yang dapat memenangi pingat di Sukan Olimpik? Bagaimana dengan atlet lain yang juga layak secara merit? Apakah kekangan atau masalah dihadapi sebagai atlet dan mahasiswa?\n",
    "\n",
    "Adakah kewujudan universiti sukan akan memberi impak besar kepada kemajuan sukan negara? Jika dirancang dan diatur dengan cekap dan sistematik, ia perkara tidak mustahil dicapai.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "string4 = \"\"\"\n",
    "KUALA LUMPUR: Tindakan sesetengah pemimpin politik mendesak supaya pengisytiharan darurat dibatalkan dan mahu Parlimen bersidang dilihat sebagai strategi untuk menyingkirkan Tan Sri Muhyiddin Yassin sebagai Perdana Menteri melalui Dewan Rakyat.\n",
    "\n",
    "Penganalisis politik dan Pakar Geostrategis, Prof Madya Dr Azmi Hassan, berkata ini bukan masanya untuk berbuat demikian, terutama ketika negara bergelut menangani pandemik COVID-19.\n",
    "\n",
    "\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(f'{cleaning(string3)}') + [1]\n",
    "s = pad_sequences([encoded], padding='post', maxlen = 1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 38s, sys: 4.19 s, total: 1min 42s\n",
      "Wall time: 7.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "l = sess.run(r[0][2], feed_dict = {X: s})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Establishment of sports universities as stated by the Head of Sports Unit of the Ministry of Higher Education, Dr Pekan Ramli and supported by Sports Development and External Recreation Specialist, Sultan Idris University of Education (UPSI), Prof Dr Md Amin Md Taaf as published by this newspaper, gives a ray of hope to the existence of such institutions. It is the dream of national athletes to pursue success in sports and excellence in academics to ensure a better future when retiring from the sports arena later. Various views, ideas, methods, evidence and suggestions were thrown by experts following the importance of sports universities that will have a big impact either at the national and international levels. Other countries have long earned profits with the existence of sports universities such as China, Korea, Japan, Taiwan, India and Vietnam. They produce university athletes who show excellent results at a high level such as the Olympics, World Championships and Asian Games. Therefore, their success needs to be used as a reference in order to develop homeland sports. If we refer to the Asian approach, the existence of important sports universities and has a positive impact in boosting sports performance is more optimal. However, if we see the European approach, sports universities are not among the important organizations or institutions given attention in contributing to medal winners. Among the issues in sports universities are high cost, location, sports infrastructure, overlap of courses with existing universities and the impact on the country's sports world to invite the question of legitimacy and relevance of its establishment. However, as a former national archery athlete and Olympian (OLY) at the 2004 Olympic Games in Athens, Greece and former students of Bukit Jalil Sports School until successful in the academic world, I found that there were several factors often an obstacle in their daily routine. Among them, the time factor that had to rush to attend lectures after completing the training session which took 15 to 20 minutes by riding motorcycles; cars (20-30 minutes) or transportation provided by the National Sports Council (MSN) to Universiti Putra Malaysia (UPM). If they demand at Universiti Teknologi MARA (UiTM) or Universiti Malaya (UM), it may be longer. Although at university is available with college facilities and sports facilities, they choose to return to MSN to undergo training with teams and coaches in the field or official training arena. This continues as long as the title of national athletes who need to ensure performance always increases from time to time without neglecting their duties as students. How good if some Bukit Jalil Sports School itself is used as a College or Universiti Sukan Malaysia because it is complete in terms of its sports infrastructure facilities and teaching and learning process (PdP) in Sports Science, Coaching, Physical Education and equivalent. The recruitment of each semester is only limited to national and international status athletes so that the desire to produce more Olympic athletes is easily realized. For example, former national jump athlete, Roslinda Samsu who is also the winner of the Asian Doha 2006 Asian Games silver medal and the recipient of the Special Award of the KPT Sports Awards 2012, had to take about nine years to complete the Bachelor of Physical Education at UPM during the 14 years involved in sports. During his tenure as national athletes and students, he also won the SEA Games Gold medal four series in a row in 2005, 2007, 2009 and 2011. Similarly, national athletes such as Leong Mun Yee (UPM); Pandalela Renong (UM); Bryan Nickson Lomas (UM); Cheng Chu Sian (UPM); Marbawi Sulaiman (UiTM) and Norasheela Khalid (UPM). If listed, maybe more people. However, it has crossed the mind why only athletes from sports diving can win medals at the Olympic Games? What about other athletes who are also eligible for merit? What are the constraints or problems faced as athletes and students? Will the existence of sports universities will have a big impact on the development of national sports? If planned and organized efficiently and systematically, it is impossible to achieve.\""
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://f000.backblazeb2.com/file/malay-dataset/test-ms-en.tar.gz\n",
    "# !tar -zxf test-ms-en.tar.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 100000)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_size = 24\n",
    "\n",
    "path = 'test'\n",
    "\n",
    "with open(os.path.join(path, 'left.txt')) as fopen:\n",
    "    left = fopen.read().split('\\n')\n",
    "    \n",
    "with open(os.path.join(path, 'right.txt')) as fopen:\n",
    "    right = fopen.read().split('\\n')\n",
    "    \n",
    "len(left), len(right)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoded = encoder.encode(left[0]) + [1]\n",
    "s = pad_sequences([encoded], padding='post', maxlen = 1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[10730,\n",
       "  1142,\n",
       "  14729,\n",
       "  221,\n",
       "  13,\n",
       "  501,\n",
       "  13,\n",
       "  960,\n",
       "  4782,\n",
       "  2005,\n",
       "  10730,\n",
       "  1142,\n",
       "  14729,\n",
       "  221,\n",
       "  13,\n",
       "  501,\n",
       "  13,\n",
       "  960,\n",
       "  4782,\n",
       "  2005,\n",
       "  30,\n",
       "  29,\n",
       "  1593,\n",
       "  25,\n",
       "  21,\n",
       "  20974,\n",
       "  1724,\n",
       "  22,\n",
       "  21,\n",
       "  20209,\n",
       "  13,\n",
       "  9752,\n",
       "  1130,\n",
       "  22,\n",
       "  12816,\n",
       "  156,\n",
       "  1432,\n",
       "  9]]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = sess.run(logits, feed_dict = {X: s}).tolist()\n",
    "results = []\n",
    "for row in p:\n",
    "    results.append([i for i in row if i not in [0, 1]])\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.97402745"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensor2tensor.utils import bleu_hook\n",
    "bleu_hook.compute_bleu(reference_corpus = [encoder.encode(right[0])], \n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 4167/4167 [3:54:49<00:00,  3.38s/it]  \n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "results = []\n",
    "for i in tqdm(range(0, len(left), batch_size)):\n",
    "    index = min(i + batch_size, len(left))\n",
    "    x = left[i: index]\n",
    "    encoded = [encoder.encode(l) + [1] for l in x]\n",
    "    batch_x = pad_sequences(encoded, padding='post', maxlen = 1024)\n",
    "    \n",
    "    p = sess.run(logits, feed_dict = {X: batch_x}).tolist()\n",
    "    result = []\n",
    "    for row in p:\n",
    "        result.append([i for i in row if i not in [0, 1]])\n",
    "    results.extend(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.58603466"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rights = [encoder.encode(r) for r in right[:len(results)]]\n",
    "bleu_hook.compute_bleu(reference_corpus = rights,\n",
    "                       translation_corpus = results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'output/model.ckpt'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saver = tf.train.Saver(tf.trainable_variables())\n",
    "saver.save(sess, 'output/model.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['bert/embeddings/word_embeddings',\n",
       " 'bert/embeddings/position_embeddings',\n",
       " 'Placeholder',\n",
       " 'bert/encoder/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/query/bias',\n",
       " 'bert/encoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/key/bias',\n",
       " 'bert/encoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_0/attention/self/value/bias',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_0/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_0/output/dense/kernel',\n",
       " 'bert/encoder/layer_0/output/dense/bias',\n",
       " 'bert/encoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/query/bias',\n",
       " 'bert/encoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/key/bias',\n",
       " 'bert/encoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/encoder/layer_1/attention/self/value/bias',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_1',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_2',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_3',\n",
       " 'bert/encoder/layer_1/attention/self/Softmax_4',\n",
       " 'bert/encoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/encoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/encoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/encoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/encoder/layer_1/output/dense/kernel',\n",
       " 'bert/encoder/layer_1/output/dense/bias',\n",
       " 'bert/encoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/decoder/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_0/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_0/attention/self/query/bias',\n",
       " 'bert/decoder/layer_0/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_0/attention/self/key/bias',\n",
       " 'bert/decoder/layer_0/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_0/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_0/attention/self/Softmax',\n",
       " 'bert/decoder/layer_0/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_0/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_0/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_0/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_0/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_0/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_0/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_0/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_0/output/dense/kernel',\n",
       " 'bert/decoder/layer_0/output/dense/bias',\n",
       " 'bert/decoder/layer_0/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_1/attention/self/query/kernel',\n",
       " 'bert/decoder/layer_1/attention/self/query/bias',\n",
       " 'bert/decoder/layer_1/attention/self/key/kernel',\n",
       " 'bert/decoder/layer_1/attention/self/key/bias',\n",
       " 'bert/decoder/layer_1/attention/self/value/kernel',\n",
       " 'bert/decoder/layer_1/attention/self/value/bias',\n",
       " 'bert/while/decoder/layer_1/attention/self/Softmax',\n",
       " 'bert/decoder/layer_1/attention/output/dense/kernel',\n",
       " 'bert/decoder/layer_1/attention/output/dense/bias',\n",
       " 'bert/decoder/layer_1/attention/output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_1/attention/encdec/query/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec/query/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec/key/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec/key/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec/value/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec/value/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec_output/dense/kernel',\n",
       " 'bert/decoder/layer_1/attention/encdec_output/dense/bias',\n",
       " 'bert/decoder/layer_1/attention/encdec_output/LayerNorm/gamma',\n",
       " 'bert/decoder/layer_1/intermediate/dense/kernel',\n",
       " 'bert/decoder/layer_1/intermediate/dense/bias',\n",
       " 'bert/decoder/layer_1/output/dense/kernel',\n",
       " 'bert/decoder/layer_1/output/dense/bias',\n",
       " 'bert/decoder/layer_1/output/LayerNorm/gamma',\n",
       " 'bert/while/decoder/layer_0/attention/self/Softmax_1',\n",
       " 'bert/while/decoder/layer_1/attention/self/Softmax_1',\n",
       " 'bert/logits',\n",
       " 'logits']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "strings = ','.join(\n",
    "    [\n",
    "        n.name\n",
    "        for n in tf.get_default_graph().as_graph_def().node\n",
    "        if ('Variable' in n.op\n",
    "        or 'Placeholder' in n.name\n",
    "        or 'logits' in n.name\n",
    "        or 'alphas' in n.name\n",
    "        or 'self/Softmax' in n.name)\n",
    "        and 'adam' not in n.name\n",
    "        and 'beta' not in n.name\n",
    "        and 'global_step' not in n.name\n",
    "        and 'gradients' not in n.name\n",
    "    ]\n",
    ")\n",
    "strings.split(',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_graph(model_dir, output_node_names):\n",
    "\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            'directory: %s' % model_dir\n",
    "        )\n",
    "\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "\n",
    "    absolute_model_dir = '/'.join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + '/frozen_model.pb'\n",
    "    clear_devices = True\n",
    "    with tf.Session(graph = tf.Graph()) as sess:\n",
    "        saver = tf.train.import_meta_graph(\n",
    "            input_checkpoint + '.meta', clear_devices = clear_devices\n",
    "        )\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess,\n",
    "            tf.get_default_graph().as_graph_def(),\n",
    "            output_node_names.split(','),\n",
    "        )\n",
    "        with tf.gfile.GFile(output_graph, 'wb') as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print('%d ops in the final graph.' % len(output_graph_def.node))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from output/model.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from output/model.ckpt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-33-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-33-9a7215a4e58a>:23: convert_variables_to_constants (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.convert_variables_to_constants`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/framework/graph_util_impl.py:277: extract_sub_graph (from tensorflow.python.framework.graph_util_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.compat.v1.graph_util.extract_sub_graph`\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 90 variables.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Froze 90 variables.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 90 variables to const ops.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Converted 90 variables to const ops.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4602 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "freeze_graph('output', strings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.tools.graph_transforms import TransformGraph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "transforms = ['add_default_attributes',\n",
    "             'remove_nodes(op=Identity, op=CheckNumerics, op=Dropout)',\n",
    "             'fold_batch_norms',\n",
    "             'fold_old_batch_norms',\n",
    "             'quantize_weights(fallback_min=-10, fallback_max=10)',\n",
    "             'strip_unused_nodes',\n",
    "             'sort_by_execution_order']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-37-ecde0d9c84a9>:4: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-37-ecde0d9c84a9>:4: FastGFile.__init__ (from tensorflow.python.platform.gfile) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.gfile.GFile.\n"
     ]
    }
   ],
   "source": [
    "pb = 'output/frozen_model.pb'\n",
    "\n",
    "input_graph_def = tf.GraphDef()\n",
    "with tf.gfile.FastGFile(pb, 'rb') as f:\n",
    "    input_graph_def.ParseFromString(f.read())\n",
    "        \n",
    "inputs = ['Placeholder']\n",
    "transformed_graph_def = TransformGraph(input_graph_def, \n",
    "                                       inputs,\n",
    "                                       ['logits'], transforms)\n",
    "\n",
    "with tf.gfile.GFile(f'{pb}.quantized', 'wb') as f:\n",
    "    f.write(transformed_graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_graph(frozen_graph_filename, **kwargs):\n",
    "    with tf.gfile.GFile(frozen_graph_filename, 'rb') as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "\n",
    "    # https://github.com/onnx/tensorflow-onnx/issues/77#issuecomment-445066091\n",
    "    # to fix import T5\n",
    "    for node in graph_def.node:\n",
    "        if node.op == 'RefSwitch':\n",
    "            node.op = 'Switch'\n",
    "            for index in xrange(len(node.input)):\n",
    "                if 'moving_' in node.input[index]:\n",
    "                    node.input[index] = node.input[index] + '/read'\n",
    "        elif node.op == 'AssignSub':\n",
    "            node.op = 'Sub'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'AssignAdd':\n",
    "            node.op = 'Add'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "        elif node.op == 'Assign':\n",
    "            node.op = 'Identity'\n",
    "            if 'use_locking' in node.attr:\n",
    "                del node.attr['use_locking']\n",
    "            if 'validate_shape' in node.attr:\n",
    "                del node.attr['validate_shape']\n",
    "            if len(node.input) == 2:\n",
    "                node.input[0] = node.input[1]\n",
    "                del node.input[1]\n",
    "\n",
    "    with tf.Graph().as_default() as graph:\n",
    "        tf.import_graph_def(graph_def)\n",
    "    return graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output/frozen_model.pb')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.48 s, sys: 244 ms, total: 5.73 s\n",
      "Wall time: 856 ms\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Maisoncelles-la-Jourdan Maisoncelles-la-Jourdan is a commune in the Calvados department of the Basse-Normandie region of northwestern France.'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "l = test_sess.run(logits, feed_dict = {x: s})\n",
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "g = load_graph('output/frozen_model.pb.quantized')\n",
    "x = g.get_tensor_by_name('import/Placeholder:0')\n",
    "logits = g.get_tensor_by_name('import/logits:0')\n",
    "test_sess = tf.InteractiveSession(graph = g)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 5.94 s, sys: 325 ms, total: 6.27 s\n",
      "Wall time: 1.2 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Maisoncelles-la-Jourdan Maisoncelles-la-Jourdan is a commune in the Calvados department of the Basse-Normandie region of northwestern France.'"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "l = test_sess.run(logits, feed_dict = {x: s})\n",
    "encoder.decode([i for i in l[0].tolist() if i > 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm -rf bigbird-small-ms-en"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
